#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
import feedparser
from jpype import *
import itertools
 
class BClassifier:
	
	t = None
	lst={}
	
	def __init__(self):
		startJVM("/opt/sun-jdk/jre/lib/i386/server/libjvm.so", "-Djava.class.path=zemberek-tum-2.0.jar")
		zerisim = JPackage('net').zemberek.erisim.Zemberek
		z = JPackage('net').zemberek.tr.yapi.TurkiyeTurkcesi
		zeris = zerisim(z())
		self.t=zeris.kokBulucu()
		
		import os
		try:
			os.remove("dirs/açık kaynak/http:----raptiye.org--blog--?p=169")
			os.remove("dirs/linux/http:----raptiye.org--blog--?p=169")
		except:
			pass
		
	def antisanitize(self, strng):
		sanitizelst = ["a", "abbr", "acronym", "address", "area", "b", "big", "blockquote", "br", "button", "caption", "center", "cite", "code", "col", "colgroup", "dd", "del", "dfn", "dir", "div", "dl", "dt", "em", "fieldset", "font", "form", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "i", "img", "input", "ins", "kbd", "label", "legend", "li", "map", "menu", "ol", "optgroup", "option", "p", "pre", "q", "s", "samp", "select", "small", "span", "strike", "strong", "sub", "sup", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "tr", "tt", "u", "ul", "var", "ta"]
		
		for x in sanitizelst:	
			
			y = "<"+x
			
			while strng.find(y) != -1:
				start = strng.find(y)
				end = strng.find(">", start)
				s = strng[start:end+1]
				
				strng = strng.replace(s, "")
		
		closetaglist = ["</"+x+">" for x in sanitizelst]
		
		for x in closetaglist:
			strng = strng.replace(x, "")
		
		strng = strng.replace("<br />", "")

		return strng
				
	def wordspliter(self, strng):
		
		lst = []
		
		lst = strng.split(" ")
		lst = [x.strip() for x in lst if x != ""]

		return lst
				
	def wordfixer(self, strng):
		strng = strng.replace('&#8242;', "'")
		strng = strng.replace('&#8217;', "'")
		strng = strng.replace('&#39;', "'")
		strng = strng.replace('"', " ")
		strng = strng.replace('...', " ")
		strng = strng.replace('•', " ")
		strng = strng.replace('‘', "")
		strng = strng.replace('^', " ")
		strng = strng.replace('~', " ")
		##strng = strng.replace("'", " ")
		strng = strng.replace('.', " ")
		strng=strng.replace("“"," ")
		strng=strng.replace("”", " ")
		strng = strng.replace(',', " ")
		strng = strng.replace('’', "'")
		strng = strng.replace(';', " ")
		strng = strng.replace(':', " ")
		strng = strng.replace('(', "")
		strng = strng.replace(')', "")
		strng = strng.replace('[', "")
		strng = strng.replace(']', "")
		strng = strng.replace('!', " ")
		strng = strng.replace('?', " ")
		strng = strng.replace('\r\n', " ")
		strng = strng.replace('\n', " ")
		strng = strng.replace('/', " ")
		strng = strng.replace('-', " ")
		strng = strng.replace('>', " ")
		strng = strng.replace('<', " ")
		strng = strng.replace('*', " ")
		strng = strng.replace('=', " ")
		strng = strng.replace('&#8230', " ")
		strng = strng.replace('&#8216', "'")
		strng = strng.replace('&#8218', " ")
		strng = strng.replace('&#8220', " ")
		strng = strng.replace('&#8221', " ")
		strng = strng.replace('&#8222', " ")
		strng = strng.replace('&#8243', " ")
		strng = strng.replace('&#038', " ")
		strng = strng.replace('&#039', " ")
		strng = strng.replace('&#8212', " ")
		strng = strng.replace('&#8211', " ")
		strng = strng.replace('&#129', " ")
		strng = strng.replace('&#141', " ")
		strng = strng.replace('&#8458', " ")
		strng = strng.replace('&#169', " ")
		strng = strng.replace('&#174', " ")
		strng = strng.replace('&#08', " ")
		strng = strng.replace('&#09', " ")
		strng = strng.replace('&#10', " ")
		strng = strng.replace('&#11', " ")
		strng = strng.replace('&#32', " ")
		strng = strng.replace('&#44', " ")
		strng = strng.replace('&#45', " ")
		strng = strng.replace('&#46', " ")
		strng = strng.replace('&#58', " ")
		strng = strng.replace('&#59', " ")
		strng = strng.replace('&#60', " ")
		strng = strng.replace('&#61', " ")
		strng = strng.replace('&#62', " ")
		strng = strng.replace('&#63', " ")
		strng = strng.replace('&amp', " ")
		strng = strng.replace('&lt', " ")
		strng = strng.replace('&gt', " ")
		strng = strng.replace('&quot', " ")
		strng = strng.replace('&lsquo', " ")
		strng = strng.replace('&rsquo', " ")
		strng = strng.replace('&sbquo', " ")
		strng = strng.replace('&bdquo', " ")
		strng = strng.replace('&lsaquo', " ")
		strng = strng.replace('&rsaquo', " ")
		strng = strng.replace('&quot', " ")
		strng = strng.replace('______________________________', " ")
		strng = strng.replace('………', " ")
		strng = strng.replace('—', " ")
		strng = strng.replace('||', " ")
		return strng
	
	def singlequotesplitter(self, word):
		if word.find("'") != -1:
			lst = word.split("'")
			if len(lst) == 1: return word
			elif len(lst) == 2: return lst[0]
			elif len(lst)>2: return lst[1]
			else: return word
		else:
			return word
	
	def train(self, wrd):
				
		rslt = None
		
		word = self.singlequotesplitter(wrd) ##take out the single quotes and evaluate the rest
		
		numbers = ('0','1','2','3','4','5','6','7','8','9')
		
		if word in numbers: return word
		
		if len(word) == 1: return rslt ## ir dont want to evaluate the single letter alphabets
		
		def rtrtip(x):
			return x.tip()
		
		try:
			z = self.t.kokBul(word)
		except:
			#print word
			return rslt
		if len(z) == 0: return word ##unresolved word, most probably a special type
		
		for x in z:
			tip = rtrtip(x)
			if tip.toString().find("BAGLAC") != -1 or tip.toString().find("EDAT") != -1 or tip.toString().find("ZAMIR") != -1: return rslt
		
		for x in z:
			tip = rtrtip(x)
			if x.toString().find("ISIM") != -1 and x.toString().find("YAL") != -1: return rslt ##eğer, her
			if tip.toString() == "ISIM" or tip.toString() == "SIFAT": ##if there is a SDK_FIIL i should use that part
				r=itertools.takewhile(lambda x: tip.toString() == "SDK_FIIL", z)
				tmp = None
				for sdkfiil in r:
					tmp = sdkfiil
					if tmp:
						return tmp.icerik()
				
				return x.icerik()
		
		for x in z:
			if tip.toString() == "OZEL" or 	tip.toString() == "SAYI":
				return x.icerik()
			
		for x in z:
			tip = rtrtip(x)
			if tip.toString() == "FIIL": return rslt
		
	def cleaner(self, dict):
		eraselst = ("evet","hayır","birçok","şöyle","daha","hiç","zaten","biraz","henüz","ne","veya","ve","hem","ki","illa","bura","de","da","için","ula","ora","ya","yani","herhalde")
		for s in eraselst:
			try:
				del(dict[s])	
			except: pass
		return dict
			
	def summaryfilescreate(self,c,lst):
		
		self.lst=lst
		
		import os
	
		workpath = os.getcwd()
	
		for root, dirs, files in os.walk('files'):
		
			if files == []:
				continue
			elif dirs == [] and files != []:
				
				os.chdir(workpath+"/"+root)
				
				for f in files:
					#print f
					#filename = f.replace("--","/")
					if self.lst.has_key(f) and self.lst[f] == True: pass
					else: continue
					testerfile = open(f+"wordlist","w+")
					sourcefile = open(f,"r")
					
					strng = sourcefile.readlines()
					st=" "
					for s in strng[3:]:
						st += s
					
					strng = strng[1]+" "+st ##take the title and the entry part to find words
					strng = c.wordfixer(strng)
					
					lst = c.wordspliter(strng)
					
					for x in lst:
						if x=="": continue
						testerfile.write(x)
						testerfile.write("\n")
					del strng	
					testerfile.close()
					sourcefile.close()
				os.chdir(workpath)
			else: continue	
		os.chdir(workpath)
		
	##test part
	##trainer, keywords detected

		for root, dirs, files in os.walk('files'):
			
			if files == []:
				continue
			elif dirs == [] and files != []:
				
				os.chdir(workpath+"/"+root)
				
				for f in files:
					trainlst = {} ## dictionary for trained words
					if f.find("wordlist") == -1: continue
					else:
						sourcefile = open(f,"r")
						trainedwords = open(f+"trainedwords","w+")
						summary = open(f+"summary", "w+")
	 
						for word in sourcefile.readlines():
							word = word.strip()
							result = c.train(word)
							if result != None: 
								trainedwords.write(result)
								try:
									trainlst[result] += 1 ## i should use this numbers 	later
								except:	
									trainlst[result] = 1 
								
							else: continue
							trainedwords.write("\n")
						
					for k,v in c.cleaner(trainlst).iteritems(): summary.write(k+" " + str(v)+"\n")
					summary.close()	
					sourcefile.close()
					trainedwords.close()
					
				os.chdir(workpath)
		os.chdir(workpath)	

	
if __name__=="__main__":
	
	import BClassifier
	c = BClassifier.BClassifier()
	
##test part
##all words writtten line by line in a file	

	import os
	
	workpath = os.getcwd()
	
	for root, dirs, files in os.walk('dirs'):
		
		if files == []:
			continue
		elif dirs == [] and files != []:
			
			os.chdir(workpath+"/"+root)
			
			for f in files:
				testerfile = open(f+"wordlist","w+")
				sourcefile = open(f,"r")
				
				strng = sourcefile.readlines()
				st=" "
				for s in strng[3:]:
					st += s
				
				strng = strng[1]+" "+st ##take the title and the entry part to find words
				strng = c.wordfixer(strng)
				
				lst = c.wordspliter(strng)
				
				for x in lst:
					if x=="": continue
					testerfile.write(x)
					testerfile.write("\n")
					
				testerfile.close()
				sourcefile.close()
			os.chdir(workpath)
		else: continue	
	os.chdir(workpath)
	
##test part
##trainer, keywords detected

	for root, dirs, files in os.walk('dirs'):
		
		if files == []:
			continue
		elif dirs == [] and files != []:
			
			os.chdir(workpath+"/"+root)
			
			for f in files:
				trainlst = {} ## dictionary for trained words
				if f.find("wordlist") == -1: continue
				else:
					sourcefile = open(f,"r")
					trainedwords = open(f+"trainedwords","w+")
					summary = open(f+"summary", "w+")
 
					for word in sourcefile.readlines():
						word = word.strip()
						result = c.train(word)
						if result != None: 
							trainedwords.write(result)
							try:
								trainlst[result] += 1 ## i should use this numbers later
							except:
								trainlst[result] = 1 
							
						else: continue
						trainedwords.write("\n")
						
					for k,v in c.cleaner(trainlst).iteritems(): summary.write(k+" " + str(v)+"\n")
					summary.close()	
					sourcefile.close()
					trainedwords.close()
					
			os.chdir(workpath)
	os.chdir(workpath)	
